package kisti.lod.classification;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.net.SocketTimeoutException;
import java.sql.SQLException;
import java.util.List;
import java.util.Vector;

import kisti.lod.classification.db.DBConnector;
import kisti.lod.classification.db.Domain;
import kisti.lod.classification.db.URI;
import kisti.lod.sindice.MySindice;
import kisti.lod.sindice.TooManyRequestsException;
import kisti.lod.sindice.query.LiveQuery;

import org.apache.log4j.Logger;

import com.hp.hpl.jena.query.Dataset;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.shared.JenaException;
import com.hp.hpl.jena.tdb.TDB;
import com.hp.hpl.jena.tdb.TDBFactory;
import com.j256.ormlite.dao.CloseableIterator;
import com.j256.ormlite.dao.Dao;
import com.j256.ormlite.dao.DaoManager;
import com.j256.ormlite.dao.GenericRawResults;
import com.j256.ormlite.stmt.UpdateBuilder;
import com.sindice.SindiceException;
import com.sindice.query.CacheQuery;
import com.sindice.result.CacheResult;

/**
 * The main method iterates over all domains and over all URIs in those domains
 * For each Domain, it extracts triples using sindice. The triples are stored locally
 * using Jena TDB
 * @author Sebastian
 *
 */
public class TripleCollector {

	public static final String LIVE_API = "live_api";
	public static final String CACHE_API = "cache_api";
	
	protected static String apiType = CACHE_API; 
	protected static MySindice sindice = new MySindice();
	protected static LiveQuery liveQuery = new LiveQuery();
	
	protected static StringBuffer foo = new StringBuffer();
	
	public static void main(String[] args) throws SQLException, IOException {
		Dao<Domain, String> domainDao = DaoManager.createDao(DBConnector.getConnectionSource(), Domain.class);
		Dao<URI, String> uriDao = DaoManager.createDao(DBConnector.getConnectionSource(), URI.class);
		Logger logger = Logger.getLogger(TripleCollector.class);
		
		//number of uris to be kept in memory before writing to disk
		int uriBufferLimit = 5;
		
		//iterate over all domains
		CloseableIterator<Domain> domainIterator = domainDao.iterator();
		while(domainIterator.hasNext()) {
			Domain domain = domainIterator.next();
			
			//dodgy quick fix to skip everything except for technology
//			if(domain.getDomainId() != 1 || domain.getDomainId() != 4) {
//				continue;
//			}
			System.out.println("DOMAIN: " + domain.getName());
			
			//create one triplestore for each domain (may be merged later)
			Dataset ds = TDBFactory.createDataset(FileFactory.getTdbDomainFolder(domain.getName()));
		    Model model = ds.getDefaultModel();
			
			//select uris that were found for the given domain using sound keywords
			//TODO: create query using the ORMLite query builder instead of using raw queries
			String query = "SELECT `u`.`uri`, `k`.`name`, `u`.`uriId` " +
					"FROM `domain` AS d " +
					"LEFT JOIN `keyword` AS k " +
					"USING ( `domainId` ) " +
					"LEFT JOIN `uri` AS u " +
					"USING ( `keywordId` ) " +
					"WHERE `d`.`domainId` = " + domain.getDomainId() +
					"  AND `k`.`sound` = 1 " +
					"  AND `u`.`triples_collected` = 0 " + 
					"ORDER BY `k`.`name`";
//					" ORDER BY RAND()" +
//					" LIMIT 5";
			GenericRawResults<String[]> rawResults = domainDao.queryRaw(query);
			List<String[]> results = rawResults.getResults();
			
			String oldKeyword = "";
			Vector<Integer> processedUriIds = new Vector<Integer>();
			
			int uriCount = 0;
			
			Model tmpModel = ModelFactory.createDefaultModel();
			
			//iterate over all URIs associated with given domain
			for(int i=0;i<results.size();i++) {
				String[] row = results.get(i);
				
				String subjectUri = row[0];
				String keyword = row[1];
				Integer uriId = Integer.valueOf(row[2]);
				if(!oldKeyword.equals(keyword)) {
					System.out.println("  - " + keyword);
					oldKeyword = keyword;
				}
				
				try {
					StringBuffer tripleBuffer = new StringBuffer();
					//add domain and searchKeyword to given URI
					String domainTriple = "<" + subjectUri + "> " +
							"<http://isrl.kisti.re.kr/ontology#domain> " +
							"<http://isrl.kisti.re.kr/ontology#" + domain.getName() + "> .";
					String keywordTriple = "<" + subjectUri + "> " + 
							"<http://isrl.kisti.re.kr/ontology#searchKeyword> " +
							"\"" + keyword + "\" . ";
					tripleBuffer.append(domainTriple + "\n");
					tripleBuffer.append(keywordTriple + "\n\n");
					
					StringBuffer sindiceTriples = getTriples(subjectUri); 
					if(sindiceTriples == null) {
						throw new SindiceException("No triples returned! " + domain.getName());
					}
					tripleBuffer.append(sindiceTriples);
					
					foo.append(tripleBuffer);
					
					//convert string to input stream for jena to parse
					byte[] bytes;
					//try to convert to utf-8
					try {
						bytes = tripleBuffer.toString().getBytes("UTF-8");
					} catch (UnsupportedEncodingException e) {
					//if conversion fails, replace every non utf-8 char with a questionmark
						String triples = tripleBuffer.toString();
						triples.replaceAll("[^(\\x20-\\x7F)]", "?");
						bytes = triples.getBytes();
					}
					InputStream is = new ByteArrayInputStream(bytes);
					tmpModel.read(is, null, "N-TRIPLE"); //parse N-Triples using JENA
					
					//add uri to list of processed uris
					processedUriIds.add(uriId);
				}
				catch (SocketTimeoutException e) {
					System.out.println("Read Timeout for: " + subjectUri);
					logger.error("Read Timeout for: " + subjectUri);
				}
				catch(JenaException e) {
					StringWriter sw = new StringWriter();
					PrintWriter pw = new PrintWriter(sw);
					e.printStackTrace(pw);
					System.out.println("Jena Parse Error: " + subjectUri);
//					System.out.println(sw.toString());
					logger.error("Jena Parse Error: \n URI: " + subjectUri + "\n" + sw.toString());
				}
				catch (SindiceException e) {
					StringWriter sw = new StringWriter();
					PrintWriter pw = new PrintWriter(sw);
					e.printStackTrace(pw);
					System.out.println("Sindice Error: " + subjectUri);
//					System.out.println(sw.toString());
					logger.error("Sindice Error: \n URI: " + subjectUri + "\n" + sw.toString());
				}
				
				//sync model and mark uri as processed after n uris
				uriCount++;
				if(uriCount >= uriBufferLimit) {
					System.out.println("--- syncing ---");
					try {
						//sync model and create new model
						model.add(tmpModel);
						TDB.sync(model);
						System.out.println(foo);
						return;
					}
					catch (Exception e) {
						StringWriter sw = new StringWriter();
						PrintWriter pw = new PrintWriter(sw);
						e.printStackTrace(pw);
						logger.error("Sync error!");
						logger.error(pw);
					}
					tmpModel.close();
					tmpModel = ModelFactory.createDefaultModel();
//					model = ds.getDefaultModel();
					//update database and mark uris as processed
					UpdateBuilder<URI, String> updateBuilder = uriDao.updateBuilder();
					updateBuilder.updateColumnValue("triples_collected", 1);
					updateBuilder.where().in("uriId", processedUriIds);
					updateBuilder.update();
						
					processedUriIds = new Vector<Integer>(); //empty processedUriVector
					uriCount = 0;
				}
			}
			
			System.out.println("--- syncing ---");
			//sync model and create new model
			model.add(tmpModel);
			TDB.sync(model);
			//update database and mark uris as processed
			UpdateBuilder<URI, String> updateBuilder = uriDao.updateBuilder();
			updateBuilder.updateColumnValue("triples_collected", 1);
			updateBuilder.where().in("uriId", processedUriIds);
			updateBuilder.update();
			
			//close model and dataset before processing the next domain
			model.close();
			tmpModel.close();
			ds.close();
		}
		domainIterator.close();
	}
	
	public static StringBuffer getTriples(String subjectUri) throws SindiceException, SocketTimeoutException {
		StringBuffer tripleBuffer = new StringBuffer();
		if(apiType == CACHE_API) {
			//query for explicit triples using sindice
			CacheQuery cacheQuery = new CacheQuery(subjectUri);
			CacheResult cacheResult = sindice.cacheQuery(cacheQuery);
			String[] triples = cacheResult.getExplicitContent();
			
			if(triples.length == 0) {
				return null;
			}
			
			//append all triples to string buffer
			for(int z=0;z<triples.length;z++) {
				tripleBuffer.append((triples[z]));
			}
			return tripleBuffer;
		}
		else {
			try {
				tripleBuffer = liveQuery.performQuery(subjectUri);
			}
			//if querying fails due to too many requests, sleep for 3 seconds and try again
			catch (TooManyRequestsException e) {
				try {
					Thread.sleep(3000);
				}
				catch (InterruptedException ie) {
				}
				tripleBuffer = liveQuery.performQuery(subjectUri);
				return tripleBuffer;
			}
			return tripleBuffer;
		}
	}
}